// rs_table.cpp

// tables suitable for language data
// all data are held in memory while running
// file format is text-readable, utf8 encoded

// todo: allow selection of rows by comparing fields in the row e.g.
// this would help to filter-out 
//   2257 asia     5 5 4781 0 5
// as proper noun although there are some occurences as lower-case


#define _CRT_SECURE_NO_DEPRECATE
#include <regex>
#include <iostream>
#include <sstream>
#include <iomanip>
#include <algorithm>
#include "rs_table.h"
#include "rs_stream.h"
#include "rs_log.h"

static const wchar_t *tbl_str_abbr = L"prop: abbreviation";
static const wchar_t *tbl_str_name = L"prop: name";
static const wchar_t *tbl_str_tag = L"prop: lang_tag";
static const wchar_t *tbl_str_col = L"columns";
static const wchar_t *tbl_str_row = L"rows";
static const wchar_t *tbl_str_numbered = L"lines_numbered";
static const wchar_t *tbl_str_col_name_pre = L"column_";
static const wchar_t *tbl_str_hash = L"# ";
static const wchar_t  tbl_chr_hash = L'#';
static const wchar_t *tbl_str_dollar = L"$ ";
static const wchar_t  tbl_chr_dollar = L'$';
static const wchar_t *tbl_str_colon = L": ";
static const wchar_t *tbl_str_dict = L"table";

enum comp_op {
    co_none,
    co_less_than,
    co_less_equal,
    co_greater_than,
    co_greater_equal,
    co_equal,
    co_different,
    co_matches
};


class rst_expression {
    // simple expression with exactly one operator, 2 usages
    // can be used as boolean expression to filter output
    // can be used as link between two tables
public:
    rst_expression (const rs_string& str, const rs_table* table1, const rs_table* table2=0);
    rst_expression (const rst_expression&);  // copy constructor
    rst_expression& operator= (const rst_expression&); // copy assignment

    ~rst_expression() { 
        if (rex_regex) {
            delete rex_regex; rex_regex=0;
        }
    }
    bool eval_cond (const rs_rix& row);
    rs_rix get_row2 (rs_rix row1) const;
    const rs_string& get_side2() const { return rex_side2; };
private:
    rs_type  rex_type;
    any_cix  rex_index1; // column number in rex_table1
    rs_string  rex_side1;

    bool rex_link;  // flag: is link between two tables
    unsigned rex_numb; // if rex_type == rt_numb4, number used in comparisons
    rs_string  rex_side2; // text after operator
     // if !rex_link and rex_type == rt_text2, regular expression used in comparisons
     // if rex_link: column name in table2
    // better: generate already regular expressions
    const tr1::wregex* rex_regex; // non-null if !rex_link && rex_type == rt_text2
    comp_op  rex_oper;
    const rs_table* rex_table1;
    const rs_table* rex_table2;
    any_cix rex_index2;  // when rex_link, column number in rex_table2
    comp_op get_comp_op (rs_string& str);
};


//static const tr1::regex_constants::match_flag_type regex_mode = 
    //boost::regex::perl;
static const tr1::wregex::flag_type regex_mode = 
    tr1::wregex::ECMAScript;

rst_expression::rst_expression (const rst_expression& rex) :
  rex_index2 () 
{
    rex_type = rex.rex_type;
    rex_index1 = rex.rex_index1;
    rex_side1 = rex.rex_side1;
    rex_link = rex.rex_link;
    rex_numb = rex.rex_numb;
    rex_side2 = rex.rex_side2;
    rex_oper = rex.rex_oper;
    rex_table1 = rex.rex_table1;
    rex_table2 = rex.rex_table2;
    rex_regex = 0;
    if (rex.rex_regex)
        rex_regex = new tr1::wregex(rex_side2, regex_mode);
}


rst_expression& rst_expression::operator= (const rst_expression& rex)
{
    if (this != &rex) {
        rex_type = rex.rex_type;
        rex_index1 = rex.rex_index1;
        rex_side1 = rex.rex_side1;
        rex_link = rex.rex_link;
        rex_numb = rex.rex_numb;
        rex_side2 = rex.rex_side2;
        rex_oper = rex.rex_oper;
        rex_table1 = rex.rex_table1;
        rex_table2 = rex.rex_table2;
        rex_index2 = rex.rex_index2;
        if (rex_regex) {
            delete rex_regex;
            rex_regex = 0;
        }
        if (rex_type == rt_text2  &&  !rex_link) {
            rex_regex = new tr1::wregex(rex_side2, regex_mode);
        }
    }
    return *this;
}


comp_op rst_expression::get_comp_op (rs_string& str)
{ // finds the predicate that is at the start of string 'str'
    // updates 'str' so that it has the operator text
    size_t len = str.length();
    if (len==0)
        return co_none;
    if (len>2)
        len = 2;
    str = str.substr (0, len);
    if (str == L"==")
        return co_equal;
    if (str == L"!=")
        return co_different;
    if (str == L">=")
        return co_greater_equal;
    if (str == L"<=")
        return co_less_equal;
    str = str.substr (0,1);
    if (str == L">")
        return co_greater_than;
    if (str == L"<")
        return co_less_than;
    if (str == L":")
        return co_matches;
    return co_none;
}


rst_expression::rst_expression (const rs_string& str, const rs_table* table1, const rs_table* table2) :
    rex_table1 (table1),
    rex_table2 (table2)
{
    rex_link = table2 != 0;
    rex_type = rt_none;
    rex_regex = 0;
    // get the column name
    if (str.length() == 0 || !is_alpha (str[0]/*, wdm_basic*/) || is_alpha (str)) {
        cout << "error: failed to get conditions starting with letter(s) "
            << "followed by non-letter character" << endl;
        return;
    }
    int i=0;
    while (is_ident_char (str[i]))
        i++;
    rex_side1 = str.substr (0, i); // text before operator

    rs_string cmp_oper_str (str.substr (rex_side1.length()));
    rex_oper = get_comp_op (cmp_oper_str);
    if (rex_oper == co_none) {
        cout << "error: unknown comparing operator after name in " << stringed(str) << endl;
        return;
    }
    rex_side2 = str.substr (rex_side1.length() + cmp_oper_str.length()); // text after operator

    rex_table1->get_any_cix (rex_side1, rex_index1, rex_type);
    if (!rex_index1.is_valid()) {
        cout << "error: column '" << stringed(rex_side1)
            << "' not found in table " << stringed (rex_table1->get_abbr()) << endl;
        return;
    }

    if (!rex_link) {
        if (rex_type == rt_numb4) { // found numeric column
            if (rex_oper == co_matches) {
                cout << "error: can't use matching operator ':' with numeric data" << endl;
                return;
            }
            rex_numb = string_to_int (rex_side2);
        }
        if (rex_type == rt_text2) { // found textual column
            if (rex_oper != co_matches) {
                cout << "error: can't use operator '" << stringed(cmp_oper_str)
                    << "' with textual data" << endl;
                return;
            }
            rex_regex = new tr1::wregex(rex_side2, regex_mode);
        }
    } else { // here code when rex_link
        rs_type type2;
        rex_table2->get_any_cix (rex_side2, rex_index2, type2);
        if (!rex_index2.is_valid()) {
            cout << "error: column '" << stringed(rex_side2)
                << "' not found in table " << stringed (rex_table2->get_abbr()) << endl;
            return;
        }
        if (rex_type != type2) {
            cout << "error: different types for columns '"
                << stringed (rex_table1->get_abbr()) << "." << stringed(rex_side1)
                << " and "
                << stringed (rex_table2->get_abbr()) << "." << stringed(rex_side2)
                << endl;
            return;
        }
        if (rex_oper != co_equal) {
            cout << "error: incorrect operator, expected '==', found "
                << stringed (cmp_oper_str) << endl;
            return;
        }

    }
}


bool rst_expression::eval_cond (const rs_rix& row)
{
    unsigned numb;
    rs_string text;
    bool result;
    tr1::match_results<rs_string::iterator> what;
    switch (rex_oper) {
    case co_none:
        return false; // todo : proclaim error
    case co_less_than:
        numb = rex_table1->get_numb (numb_cix(rex_index1), row);
        return numb < rex_numb;
    case co_less_equal:
        numb = rex_table1->get_numb (numb_cix(rex_index1), row);
        return numb <= rex_numb;
    case co_greater_than:
        numb = rex_table1->get_numb (numb_cix(rex_index1), row);
        return numb > rex_numb;
    case co_greater_equal:
        numb = rex_table1->get_numb (numb_cix(rex_index1), row);
        return numb >= rex_numb;
    case co_equal:
        numb = rex_table1->get_numb (numb_cix(rex_index1), row);
        return numb == rex_numb;
    case co_different:
        numb = rex_table1->get_numb (numb_cix(rex_index1), row);
        return numb != rex_numb;
    case co_matches:
        text = rex_table1->get_text (text_cix(rex_index1), row);
        result = regex_match(text.begin(), text.end(), what, *rex_regex, tr1::regex_constants::match_default);
        return result;
    default:
        return false;
    }
}


rs_rix rst_expression::get_row2 (rs_rix row1) const
{
    if (rex_type == rt_text2) {
        rs_string value1 = rex_table1->get_text (text_cix(rex_index1), row1);
        bool several;
        return rex_table2->get_text_row_any (value1, text_cix(rex_index2), several);
        // todo: if there are several matches, report error
    }
    // no need for linking numeric values
    return rs_rix(rs_rix::none());
}

bool rs_table::check_var (const rs_string& line, const rs_string& var_name, rs_string& var_value)
{
    rs_string var_prefix=tbl_str_dollar +var_name+L": ";
    size_t pos = line.find (var_prefix);
    if (pos == 0) {
        var_value = line.substr (var_prefix.length());
        return true;
    }
    return false;
}


bool rs_table::check_var (const rs_string& line, const rs_string& var_name, size_t& var_value)
{
    rs_string str_value;
    if (check_var (line, var_name, str_value)) {
        var_value = string_to_int (str_value);
        return true;
    }
    return false;
}


bool rs_table::read_line (const rs_string& input_line, bool update_hashes, bool last_all)
{
    // parses text from a single line into columns of table
    wistringstream wiss (input_line);
    if (rst_numbered) {
        unsigned ix;
        wiss >> ix; // ix is not used for processing, it is optional for better readability of table file
    }
    for (unsigned i=0; i<rst_data_cols; i++) {
        if (rst_table[i].rsc_type == rt_text2) {
            rs_string word;
            if (last_all && i+1 == rst_data_cols) {
                while (!wiss.eof()) {
                    rs_string item;
                    wiss >> item;
                    if (word.length() > 0)
                        word += L"_";
                    word += item;
                }
            } else
                wiss >> word;
            if (word == L"\"\"") // empty string
                word = L"";
            append_text (word, text_cix (i), update_hashes);
        } else { // numeric value
            unsigned nvalue = 0;
            wiss >> nvalue;
            // todo check conversion
            append_numb (nvalue, numb_cix(i));
        }
    }
    return true; // right now, only success is reported
}

bool rs_table::read_one (const rs_string& rst_fname, bool update_hashes)
{
    // appends data from one file to a table
    // mostly tested with initially empty table
    // input file is in internal format
    // rst_fname can optionally contain case-conversion operator:
    // if rst_fname starts with "lc:", every line is converted to lower-case
    // if rst_fname starts with "uc:", every line is converted to upper-case
    // this operator is then removed from file name
    rs_string line;
    unsigned og_row=0, i;
    bool var_set, preallocated=false;
    bool line_to_lower (false), line_to_upper (false);
    rs_string case_conversion;
    rs_string actual_rst_fname (rst_fname);
    if (lower_case (rst_fname.substr(0,3)) == L"lc:") {
        line_to_lower = true;
        actual_rst_fname = rst_fname.substr (3);
        case_conversion = L", converting to upper-case";
    } else if (lower_case (rst_fname.substr(0,3)) == L"uc:") {
        line_to_upper = true;
        actual_rst_fname = rst_fname.substr (3);
        case_conversion = L", converting to lower-case";
    }
    rs_istream ifs(actual_rst_fname);
    if (!ifs.good()) {
        rst_valid = false;
        return false;
    }
    rs_log_file (L"Reading table " + actual_rst_fname
        + L", updating_hashes=" + rs_stringed (update_hashes)
        + case_conversion
        + L" ... ");
    size_t count_lines=1;
    size_t all_col=0;
    rs_string str_hash (tbl_str_hash);
    while (!ifs.eof()) {
        ifs.read_line (line);
        if (line.length() == 0)
            continue;
        if (line[0]==tbl_chr_hash)
            continue;
        if (line[0]==tbl_chr_dollar) {
            // process some variables
            var_set = check_var (line, tbl_str_name, rst_name)
                || check_var (line, tbl_str_abbr, rst_abbr)
                || check_var (line, tbl_str_tag, rst_tag)
                || check_var (line, tbl_str_col, all_col)
                || check_var (line, tbl_str_row, og_row)  // expected number of rows, actual can differ
                || check_var (line, tbl_str_numbered, count_lines);
            rst_numbered = count_lines != 0;
            if (var_set)
                continue;
            if (line == tbl_chr_dollar + tbl_str_dict && !preallocated) {
                // preallocate space for data columns
                preallocated = true;
                for (i=0; i<rst_data_cols; i++) {
                    if (get_col_type (any_cix(i)) == rt_text2)
                        rst_table[i].rsc_text.reserve (og_row);
                    else
                    if (get_col_type (any_cix(i)) == rt_numb4)
                        rst_table[i].rsc_numb.reserve (og_row);
                }
            }
            rs_string col_desc;
            for (i=0; i<all_col; i++) {
                if (check_var (line, tbl_str_col_name_pre + rs_stringed (i), col_desc))
                    break;
            }
            if (col_desc!=L"") {
                rs_string type_str = nth_substring (col_desc, 1, L",");
                wchar_t type_desc = type_str[0];
                bool hashed = type_str.substr(1) == L"#";
                unsigned min_width= string_to_unsigned(nth_substring (col_desc, 2, L","));
                col_desc = nth_substring (col_desc, 0, L",");
                any_cix added_cix;
                if (type_desc==L'N')
                    added_cix = add_numb_col (col_desc, min_width, true);
                else
                    added_cix = add_text_col (col_desc, hashed, min_width, true);
            }
            continue;
        }
        if (line_to_upper)
            to_upper (line);
        else if (line_to_lower)
            to_lower (line);
        read_line (line, update_hashes, false);
    }
    rs_log_file (L"done, " + rs_stringed(rst_data_rows) + L" rows, "
        + rs_stringed(rst_data_cols) + L" columns");
    return true;
}


bool rs_table::import_table (const rs_string& in_fname, const rs_string& skip_prefix, bool update_hashes)
{
    rs_istream ifs (in_fname);
    if (!ifs.good())
        return false;
    rst_numbered = false;
    unsigned skip_len = skip_prefix.length();
    while (!ifs.eof()) {
        rs_string input_line;
        ifs.read_line (input_line);
        if (skip_len > 0 && input_line.substr (0, skip_len) == skip_prefix)
            continue;
        trim_spaces (input_line);
        read_line (input_line, update_hashes, true);
    }
    return true;
}


void rs_table::create_hashes ()
{
    rs_log_file (L"Creating hashes");
    for (unsigned cix=0; cix<rst_data_cols; cix++) {
        if (get_col_type (any_cix (cix)) == rt_text2 &&
            get_col_hash (any_cix (cix)) && 
            rst_table[cix].rsc_hash.size() == 0) {
            text_cix text_ix (cix);
            for (size_t rix=0; rix < rst_data_rows; rix++) {
                rst_table[cix].rsc_hash.insert ( text_pair(
                    rst_table[cix].rsc_text[rix], 
                    rix));
            }
        }
    }
}


bool rs_table::read (const rs_string& rst_fname_list, bool update_hashes)
{
    init();
    vector<rs_string> rst_fname_vec;
    string_to_vec_delim (rst_fname_list, rst_fname_vec, L'+');
    if (rst_fname_vec.size() == 0) {
        rst_valid = false;
        return false;
    }
    read_one (rst_fname_vec[0], false);
    if (!rst_valid)
        return false;
    size_t tix;
    for (tix=1; tix<rst_fname_vec.size(); tix++) {
        rs_table rst_add;
        rst_add.read_one (rst_fname_vec[tix], false);
        if (!rst_add.is_valid()) {
            rst_valid = false;
            return false;
        }
        if (!add_all_rows (rst_add, get_col_count())) {
            rst_valid = false;
            return false;
        }
    }
    if (update_hashes)
        create_hashes();
    return true;
}


rs_rix rs_table::append_text (const rs_string& text, const text_cix cix, bool update_hashes) {
    // to specified 'cix', adds 'text'
    // for hashed columns, adds entry into associated hash
    // and updates 'rst_data_rows' according to the length of this cix
    // returns row index of added text
    rst_table[cix].rsc_text.push_back (text);
    rst_data_rows = (unsigned) (rst_table[cix].rsc_text.size());
    if (rst_table[cix].rsc_hashed && update_hashes)
        rst_table[cix].rsc_hash.insert ( text_pair(text, (unsigned) (rst_data_rows-1)));
    return rs_rix(rst_data_rows-1);
}


rs_rix rs_table::append_numb (unsigned numb, const numb_cix column) {
    // to specified 'column', adds 'numb'er
    // and updates 'rst_data_rows' according to the length of this column
    rst_table[column].rsc_numb.push_back (numb);
    rst_data_rows = (unsigned) (rst_table[column].rsc_numb.size());
    return rs_rix(rst_data_rows-1);
}

unsigned rs_table::inc_numb (numb_cix cix, rs_rix rix)
{
    return ++rst_table[cix].rsc_numb[rix];
}


unsigned rs_table::incr_numb (unsigned numb, numb_cix cix, rs_rix rix)
{
    return rst_table[cix].rsc_numb[rix] += numb;
}


bool rs_table::parse_column_list (const rs_string& columns, vector<any_cix>& col_vec) const
{
    col_vec.clear();
    size_t spec_col; // number of specified columns
    unsigned i;
    if (columns != L"" && columns!=L"*" && columns!=L".") {
        vector<rs_string> col_name_vec;
        string_to_vec_delim (columns, col_name_vec, L',');
        spec_col = col_name_vec.size();
        for (i=0; i<spec_col; i++) {
            rs_string col_str = col_name_vec[i];
            to_lower (col_str);
            trim_all_non_ident_chars (col_str);
            any_cix cdata = get_any_cix (col_str);
            if (cdata.is_valid()) {
                col_vec.push_back (cdata);
                continue;
            }
            rs_report_error (L"unknown column name: ", col_str);
            return false;
        }
    } else {
        spec_col = rst_data_cols;
        for (i=0; i<spec_col; i++)
            col_vec.push_back (any_cix(i));
    }
    return true;
}


bool rs_table::parse_and_sort_column (
    const rs_string& sort_order_str,
    bool& sorting_present,
    vector<unsigned>& row_order)
{
    row_order.clear();
    rs_rix rix;
    any_cix og_sort_column; // used while storing data
    bool og_sort_ascending;
    rs_string sort_col_name;
    sorting_present=sort_order_str != L"" && sort_order_str != L".";
    if (sorting_present) {
        rs_string sort_direction = sort_order_str.substr(0,1);
        // determine the sorting order and check data
        if (sort_direction == L"+")
            og_sort_ascending = true;
        else if (sort_direction == L"-")
            og_sort_ascending = false;
        else {
            rs_report_error (L"unknown sort-direction operator, expected either '+' or '-': operator=",
                sort_direction);
            sorting_present = false;
            return false;
        }
        sort_col_name = sort_order_str.substr (1);
        og_sort_column = get_any_cix (sort_col_name);
        if (!og_sort_column.is_valid()) {
            rs_report_error (L"unknown sort column: ", sort_order_str.substr (1));
            sorting_present = false;
            return false;
        }
    }
    if (sorting_present && rst_table[og_sort_column].rsc_type == rt_text2) {
        // sorting by textual column
        typedef pair<rs_string, size_t> text_row;
        vector<text_row> order_text;
        order_text.resize (rst_data_rows);
        text_cix tcix = get_text_cix (sort_col_name);
        for (rix=rs_rix::begin(); rix<rst_data_rows; rix++) {
            order_text[rix].first = get_text (tcix, rix);
            order_text[rix].second = (unsigned) rix;
        }
        stable_sort (order_text.begin(), order_text.end());
        row_order.resize (rst_data_rows);
        for (rix=rs_rix::begin(); rix<rst_data_rows; rix++)
            row_order[rix] = order_text[rix].second;
    }
    if (sorting_present && rst_table[og_sort_column].rsc_type == rt_numb4) {
        // sorting by numerical column
        typedef pair<int, size_t> numb_row;
        vector<numb_row> order_numb;
        order_numb.resize (rst_data_rows);
        numb_cix ncix = get_numb_cix (sort_col_name);
        for (rix=rs_rix::begin(); rix<rst_data_rows; rix++) {
            order_numb[rix].first = get_numb (ncix, rix);
            order_numb[rix].second = (unsigned) rix;
        }
        stable_sort (order_numb.begin(), order_numb.end());
        row_order.resize (rst_data_rows);
        for (rix=rs_rix::begin(); rix<rst_data_rows; rix++)
            row_order[rix] = (unsigned) order_numb[rix].second;
    }
    if (sorting_present && row_order.size() == 0  &&  rst_data_rows>0) {
        rs_report_error (L" (internal continuable): sorting specified, no order found");
        sorting_present = false;
    }
    if (sorting_present && !og_sort_ascending) {
        reverse (row_order.begin(), row_order.end());
    }
    return true;
}


void rs_table::parse_and_evaluate_conditions (
    const rs_string& conditions,
    bool& cond_present,
    vector<bool>& out_row_vec,
    unsigned& out_row_cnt)
{
    vector<rs_string> cond_vec;
    if (conditions!= L".")
        string_to_vec_delim (conditions, cond_vec, L',');
    out_row_cnt=0;
    cond_present = cond_vec.size() > 0; // row selection conditions
    size_t i;
    rs_rix rix;
    if (cond_present) {
        // calculation of row numbers that will be included in the output
        // for every row in the rs_table, get its output number out_row_vec[i]
        // or set it to -1 if not included
        vector <rst_expression> opv;
        for (i=0; i<cond_vec.size(); i++) {
            rst_expression op (cond_vec[i], this);
            opv.push_back (op);
        }
        out_row_vec.resize (rst_data_rows);
        bool cond_passed;
        for (rix=rs_rix::begin(); rix<rst_data_rows; rix++) {
            cond_passed = true;
            for (i=0; i<opv.size(); i++) {
                if (!opv[i].eval_cond (rix)) {
                    cond_passed = false;
                    break;
                }
            }
            out_row_vec[rix] = cond_passed;
            if (cond_passed)
                out_row_cnt++;
        }
    } else {
        out_row_cnt = (unsigned) get_row_count();
    }
}


bool rs_table::write (
    const rs_string& rst_fname,
    const rs_string& columns,
    const rs_string& conditions,
    const rs_string& sort_order_str,
    rs_rix max_row_index,
    bool native)
{
    // stores a table to file in native format (native==true)
    // exports a table in tab-separated format (native = false)
    // if 'columns' is non-empty, only columns listed (comma-separated) are outputted
    // if 'conditions' is non-empty, only rows matching them are outputted
    // if 'sort_order_str' is non-empty and not ".", is specifies sorting on that column
      // first character is '+' for ascending or '-' for descending sorting
    // if 'max_row_index' >= 0, at most so many rows are outputted
    rs_log_file (L"Storing table " + rst_fname + L" ... ");

    // columns to be outputted
    vector<any_cix> col_vec;
    parse_column_list (columns, col_vec);
    size_t spec_col = col_vec.size();

    // order of rows
    vector<unsigned> row_order;
    bool sorting_present;
    parse_and_sort_column (sort_order_str, sorting_present, row_order);

    vector<bool> out_row_vec;
    bool cond_present;
    unsigned out_row_cnt;
    parse_and_evaluate_conditions (conditions, cond_present, out_row_vec, out_row_cnt);

    rs_ostream ofs(rst_fname);
    if (!ofs.good())
        return false;
    unsigned i, ix;
    rs_rix act_rows (out_row_cnt);
    if (max_row_index>=0 && act_rows>max_row_index)
        act_rows = max_row_index;
    rs_string str_dollar (tbl_str_dollar);
    if (native) {
        ofs.write_line (str_dollar + L"Respell table version: 1");
        ofs.write_line (str_dollar + L"Encoding: utf8");
        ofs.write_line (str_dollar + tbl_str_abbr + tbl_str_colon + rst_abbr);
        ofs.write_line (str_dollar + tbl_str_name + tbl_str_colon + rst_name);
        ofs.write_line (str_dollar + tbl_str_tag + tbl_str_colon + rst_tag);    
        ofs.write_line (str_dollar + tbl_str_col + tbl_str_colon + rs_stringed(col_vec.size()));
        ofs.write_line (str_dollar + tbl_str_row + tbl_str_colon + rs_stringed(act_rows));
        ofs.write_line (L"# Value for \"rows\" is used to preallocate space for table data");
        ofs.write_line (L"# and can be approximate. Exact values allow faster program execution.");
        ofs.write_line (str_dollar + tbl_str_numbered + tbl_str_colon + 
            (rst_numbered? L"1" : L"0"));
    }
    {
        wostringstream oss;
        for (i=0; i<spec_col; i++) {
            const rs_string& column_name = rst_table[col_vec[i]].rsc_name;
            if (native) {
                wchar_t type_desc= rst_table[col_vec[i]].rsc_type == rt_text2? L'T' : L'N';
                rs_string hash_desc= rst_table[col_vec[i]].rsc_hashed? L"#" : L"";
                ofs.write_line (str_dollar + tbl_str_col_name_pre + rs_stringed (i) + 
                    tbl_str_colon + rst_table[col_vec[i]].rsc_name
                    + L"," + type_desc + hash_desc
                    + L"," + rs_stringed (rst_table[col_vec[i]].rsc_min_width));
            } else
                oss << (i>0? L"\t" : L"") << column_name;
        }
        if (!native)
            ofs.write_line (oss.str());
    }

    rs_string value;
    unsigned nvalue;

    if (native) {
        ofs.write_line (L"");
        ofs.write_line (rs_string(tbl_str_dollar) + tbl_str_dict);
    }
    size_t ixx = 0;  // actual line numbers: so many rows written so far
    unsigned row_no;   // tried line number, subject to sorting and row conditions
    for (row_no=0; row_no<rst_data_rows; row_no++) {
        if (max_row_index >= 0 && ixx >= (unsigned) max_row_index)
            break;
        if (sorting_present)
            ix = row_order[row_no];
        else
            ix = row_no;
        if (cond_present &&  !out_row_vec[ix])
            continue;
        wostringstream woss;
        if (rst_numbered && native) {
            woss << setw(6) << ixx;
        }
        ixx++;
        for (i=0; i<spec_col; i++) {
            unsigned min_width = rst_table[col_vec[i]].rsc_min_width;
            if (native) {
                if (i>0 || rst_numbered)
                    woss << L" ";
            } else {
                if (i>0)
                    woss << L"\t";
            }
            if (rst_table[col_vec[i]].rsc_type == rt_text2) {
                value = L"";
                if (ix < rst_table[col_vec[i]].rsc_text.size())
                    value = rst_table[col_vec[i]].rsc_text[ix];
                if (native) {
                    if (value==L"")
                        value=L"\"\"";
                    woss << setw(min_width) << left << value;
                } else
                    woss << L"\"" << value << L"\"";
            } else {
                nvalue = 0;
                if (ix < rst_table[col_vec[i]].rsc_numb.size())
                    nvalue = (unsigned) rst_table[col_vec[i]].rsc_numb[ix];
                if (native)
                    woss << setw(min_width) << right << nvalue;
                else
                    woss << nvalue;
            }
        }
        ofs.write_line (woss.str());
    }
    rs_log_file (L" done, "
        + rs_stringed (act_rows) + L" rows, "
        + rs_stringed (spec_col) + L" columns");
    return true;
}


bool rs_table::update_table (const rs_table& rst_add, const rs_string& link, const rs_string& columns)
{
    rst_expression rex_link (link, this, &rst_add);
    vector<any_cix> col_add_vec;
    rst_add.parse_column_list (columns, col_add_vec);
    size_t add_col_size = col_add_vec.size();
    vector<any_cix> col_this_vec;
    parse_column_list (columns, col_this_vec);
    size_t this_col_size = col_add_vec.size();
    if (add_col_size != this_col_size) {
        rs_report_error (L"not all columns found");
        return false;
    }
    size_t cix;
    rs_rix rix, rix_add;
    for (cix=0; cix<this_col_size; cix++) {
        if (!col_add_vec[cix].is_valid()) {
            rs_report_error (L"column not found in source: ", rs_stringed (cix));
            return false;
        }
        if (!col_this_vec[cix].is_valid()) {
            rs_report_error (L"column not found in target: ", rs_stringed (cix));
            return false;
        }
        if (rst_add.get_col_type (col_add_vec[cix])  !=  get_col_type (col_this_vec[cix])) {
            rs_report_error (L"columns not of the same type: ", rs_stringed (cix));
            return false;
        }
    }
    // let's check that link column is hashed
    rs_string link_side2 = rex_link.get_side2();
    any_cix side2_cix = rst_add.get_any_cix (link_side2);
    if (!rst_add.get_col_hash (side2_cix)) {
        rs_report_error (L"Linking column from right-side table should be hashed, but it is not: ",
            L"column_name=" + link_side2);
        return false;
    }

    for (rix=rs_rix::begin(); rix< rst_data_rows; rix++) {
        rix_add = rex_link.get_row2 (rix);
        if (!rix_add.is_valid())
            continue;
        for (cix=0; cix<this_col_size; cix++) {
            if (get_col_type (col_this_vec[cix]) == rt_text2) {
                rst_table[col_this_vec[cix]].rsc_text[rix] = 
                    rst_add.rst_table[col_add_vec[cix]].rsc_text[rix_add];
            } else {
                rst_table[col_this_vec[cix]].rsc_numb[rix] = 
                    rst_add.rst_table[col_add_vec[cix]].rsc_numb[rix_add];
            }
        }
    }
    return true;
}


bool rs_table::join_table (const rs_table& rst_add, const rs_string& link, const rs_string& columns)
{
    // adds specified 'columns' from table 'rst_add' that is linked with expression 'link'
    vector<any_cix> col_vec;
    rst_add.parse_column_list (columns, col_vec);
    size_t spec_col = col_vec.size();
    for (unsigned i=0; i<spec_col; i++) {
        any_cix cix = col_vec[i];
        add_any_col (rst_add.get_col_name (cix),
            rst_add.get_col_type (cix),
            rst_add.get_col_hash (cix),
            rst_add.get_col_min_width (cix));
    }
    return update_table (rst_add, link, columns);
}


bool rs_table::compatible_columns (const rs_table& other, unsigned col_count) const
{
    // returns true if the first col_count columns in the tables are
    // of the same type
    if (get_col_count() < col_count || 
        other.get_col_count() < col_count) {
        return false; // different number of columns
    }
    for (unsigned cix=0; cix < col_count; cix++) {
        if (get_col_type (any_cix (cix)) != other.get_col_type (any_cix (cix)))
            return false;
    }
    return true;
}


bool rs_table::get_columns_from (const rs_table& source)
{
    if (rst_data_cols>0) {
        rs_report_error (L"get_columns_from assumes that table has no columns yet");
        return false;
    }
    clear();
    for (any_cix cix=any_cix(0); cix<source.get_col_count(); cix++)
        add_any_col (
            source.get_col_name (cix),
            source.get_col_type (cix),
            source.get_col_hash (cix),
            source.get_col_min_width (cix));
    return true;
}


void rs_table::reserve (size_t n)
{
    for (unsigned cix=0; cix<rst_data_cols; cix++) {
        if (get_col_type (any_cix (cix)) == rt_text2) {
            rst_table[cix].rsc_text.reserve (n);
        }
        if (get_col_type (any_cix (cix)) == rt_numb4) {
            rst_table[cix].rsc_numb.reserve (n);
        }
    }
}



void rs_table::clear ()
{
    for (size_t cix=0; cix<rst_max_col; cix++) {
        rst_table[cix].rsc_text.clear();
        rst_table[cix].rsc_numb.clear();
        rst_table[cix].rsc_hash.clear();
    }
    rst_data_rows = 0;
}


void rs_table::add_empty_rows (unsigned row_count)
{
    unsigned cix;
    unsigned new_data_rows = rst_data_rows + row_count;
    for (cix=0; cix<rst_data_cols; cix++) {
        if (get_col_type (any_cix (cix)) == rt_text2) {
            rst_table[cix].rsc_text.resize (new_data_rows);
        }
        if (get_col_type (any_cix (cix)) == rt_numb4) {
            rst_table[cix].rsc_numb.resize (new_data_rows);
        }
    }
    rst_data_rows = new_data_rows;
}

bool rs_table::add_table_rows (const rs_table& source, const rs_rix& begin, const rs_rix& end, unsigned col_count)
{
    // adds specified rows from from table 'source' of compatible columns
    if (!compatible_columns (source, col_count)) {
        rs_report_error (L"adding rows from table, but tables don't have the initial columns of the same type");
        return false;
    }
    rs_rix end_rix;
    if (end.is_valid()) { // is 'end' is specified
        if (end > source.get_row_count()) {
            rs_report_error (L"adding rows from table, but end of range is too high");
            return false;
        } else {
            end_rix = end;
        }
    } else {
        end_rix = source.get_row_count();
    }
    if (begin > end_rix) {
        rs_report_error (L"adding rows from table, but start of range is larger than end of range");
        return false;
    }
    unsigned add_data_rows = end_rix - begin;
    unsigned new_data_rows = rst_data_rows + add_data_rows;
    for (unsigned cix=0; cix<col_count; cix++) {
        if (get_col_type (any_cix (cix)) == rt_text2) {
            rst_table[cix].rsc_text.resize (new_data_rows);
            text_cix text_ix (cix);
            for (rs_rix rix=rs_rix::begin(); rix < add_data_rows; rix++)
                set_text (source.get_text (text_ix, rs_rix(begin+rix)), 
                    text_ix, rs_rix(rst_data_rows+rix));
        }
        if (get_col_type (any_cix (cix)) == rt_numb4) {
            rst_table[cix].rsc_numb.resize (new_data_rows);
            numb_cix numb_ix (cix);
            for (rs_rix rix=rs_rix::begin(); rix < add_data_rows; rix++)
                set_numb (source.get_numb (numb_ix, rs_rix(begin+rix)), 
                    numb_ix, rs_rix(rst_data_rows+rix));
        }
    }
    rst_data_rows = new_data_rows;
    return true;
}


any_cix rs_table::add_any_col (
    const rs_string& column_name,
    rs_type col_type,
    bool hashed,
    unsigned min_width,
    bool allow_existing)
{
    // adds (or verifies, if allow_existing and already present) a column
    // also clears all the data
    // returns column index
    any_cix pre_cix;
    any_cix this_cix;
    if (allow_existing)
        pre_cix = get_any_cix (column_name);
    if (pre_cix.is_valid()) {
        // found existing column, check types
        if (col_type != get_col_type (pre_cix)) {
            rs_report_error (L"adding column " + column_name
                + L" that already exists, but of different type");
            return any_cix (any_cix::none);
        }
        this_cix = pre_cix;
        rs_log_file (L"using already existing column: ", L"name="+column_name
            +L", index="+rs_stringed(this_cix));
    } else {
        if (rst_data_cols>=rst_max_col  ||  col_type == rt_none) {
            return any_cix (any_cix::none);
        }
        this_cix = any_cix (rst_data_cols);
    }
    rst_table[this_cix].rsc_name = column_name;
    rst_table[this_cix].rsc_hashed = hashed;
    rst_table[this_cix].rsc_type = col_type;
    rst_table[this_cix].rsc_min_width = min_width;
    if (col_type == rt_text2)
        rst_table[this_cix].rsc_text.resize (rst_data_rows, L"");
    else if (col_type == rt_numb4)
        rst_table[this_cix].rsc_numb.resize (rst_data_rows, 0);
    if (!pre_cix.is_valid())
        rst_data_cols++;
    return this_cix;
}


bool rs_table::add_columns (const rs_string& columns)
{
    // adds columns to the table according to their textual representation
    // e.g. columns=L"t0,T#,10;t1,N,5" adds two columns
    rs_string_v column_v;
    string_to_vec (columns, column_v, L";");
    unsigned col_size = column_v.size();
    unsigned i;
    for (i=0; i<col_size; i++) {
        rs_string col_name (nth_substring (column_v[i], 0, L","));
        rs_string col_type (nth_substring (column_v[i], 1, L","));
        unsigned col_width (string_to_unsigned (nth_substring (column_v[i], 2, L",")));
        to_upper (col_name);
        to_upper (col_type);
        any_cix cix;
        if (col_type==L"N")
            cix = add_any_col (col_name, rt_numb4, false, col_width);
        else if (col_type==L"T#")
            cix = add_any_col (col_name, rt_text2, true, col_width);
        else if (col_type==L"T")
            cix = add_any_col (col_name, rt_text2, false, col_width);
        else {
            rs_report_error (L"Unrecognized type of column specification: " + column_v[i]);
            return false;
        }
        if (!cix.is_valid()) {
            rs_report_error (L"Couldn't add column (duplicate or bad name, too many): " + column_v[i]);
            return false;
        }
    }
    return true;
}


// 'column' is semicolon-separated list of textual column names
// that neees to be converted to either uppercase or lowercase, depending on 'to_uppercase'
bool rs_table::change_case (const rs_string& columns, bool to_uppercase)
{
    rs_string_v column_v;
    string_to_vec (columns, column_v, L";");
    unsigned col_size = column_v.size();
    unsigned i;
    for (i=0; i<col_size; i++) {
        rs_string col_name (column_v[i]);
        to_upper (col_name);
        text_cix cix = get_text_cix (col_name);
        if (!cix.is_valid()) {
            rs_report_error (L"Not found textual column with this name: " + col_name);
            return false;
        }
        unsigned this_column_size = rst_table[cix].rsc_text.size();
        rs_rix rix;
        if (to_uppercase) {
            for (rix=rs_rix::begin(); rix< this_column_size; rix++) {
                to_upper (rst_table[cix].rsc_text[rix]);
            }
        } else {
            for (rix=rs_rix::begin(); rix< this_column_size; rix++) {
                to_lower (rst_table[cix].rsc_text[rix]);
            }
        }
    }
    return true;
}


vector<rs_rix> rs_column::get_text_row_all (const rs_string& text) const
{
    // returns the vector of all the occurences of specified text
    std::pair<text_hash::const_iterator, text_hash::const_iterator> p =
        rsc_hash.equal_range(text);
    rs_rix rix;
    vector<rs_rix> rix_vec;
    for (text_hash::const_iterator i = p.first; i != p.second; ++i) {
        rix_vec.push_back (rs_rix(i->second));
    }
    return rix_vec;
}


unsigned rs_column::get_text_row_count (const rs_string& text) const
{
    // returns the count of all the occurences of specified text
    std::pair<text_hash::const_iterator, text_hash::const_iterator> p =
        rsc_hash.equal_range(text);
    unsigned result=0;
    for (text_hash::const_iterator i = p.first; i != p.second; ++i) {
        result++;
    }
    return result;
}


rs_rix rs_column::get_text_row_any (const rs_string& text, bool& several) const {
    // returns valid row index if text is found in table in specified column cix
    // if more than one such text exist, several=true and function returns any one of such words
    // if exactly one is found, several=false
    // if not found, returns invalid row index
    several = false;
    std::pair<text_hash::const_iterator, text_hash::const_iterator> p =
        rsc_hash.equal_range(text);
    rs_rix rix;
    for (text_hash::const_iterator i = p.first; i != p.second; ++i) {
        int row_ix = i->second;
        if (!rix.is_valid()) {
            // first occurence of such text
            rix = rs_rix(row_ix);
        } else {
            // second occurence
            several = true;
            break;
            // no need to check for other occurences
        }
    }
    return rix;
}


rs_rix rs_table::get_text_row_freq (
    const rs_string& text,
    text_cix search_cix,
    numb_cix freql_cix,
    bool& several) const {
    // returns valid row index if 'text' is found in table in specified column 'search_cix'
    // if more than one such text exist, several=true and function returns
    // the one that has the lowest value in column 'freql_cix'
    // if exactly one is found, several=false
    // if not found, returns invalid row index
    several = false;
    std::pair<text_hash::const_iterator, text_hash::const_iterator> p =
        rst_table[search_cix].rsc_hash.equal_range(text);
    rs_rix rix, rix_new;
    unsigned freq = 0, freq_new;
    for (text_hash::const_iterator i = p.first; i != p.second; ++i) {
        if (!rix.is_valid()) {
            // first occurence of such text
            rix = rs_rix(i->second);
            freq = get_numb (freql_cix, rix);
        } else {
            // next occurences
            several = true;
            rix_new = rs_rix(i->second);
            freq_new = get_numb (freql_cix, rix_new);
            if (freq_new < freq)
                rix = rix_new;
        }
    }
    return rix;
}


bool rs_table::translate_one (
    const rs_string &src,
    text_cix src_cix,
    numb_cix freql_cix,
    text_cix tgt_cix,
    rs_string &tgt,
    bool& several) 
    const {
    // translates word 'src' (in column src_cix) to 'tgt' (in column tgt_cix)
    // returns true if word 'src' is found in table
    // in this case, it sets 'tgt' to converted form
    // and 'several' to true if more than one translation exist
    // otherwise, it returns false and sets 'tgt' to 'src'
    rs_rix rix = get_text_row_freq (src, src_cix, freql_cix, several);
    if (rix.is_valid()) {
        tgt = rst_table[tgt_cix].rsc_text[rix];
        return true;
    }
    tgt = src;
    return false;
}



any_cix rs_table::get_any_cix (const rs_string& col_name) const {
    unsigned i;
    rs_string searched_name (col_name);
    to_lower (searched_name);
    for (i=0; i<rst_data_cols; i++) {
        rs_string this_name (rst_table[i].rsc_name);
        to_lower (this_name);
        if (this_name == searched_name)
            return any_cix(i);
    }
    return any_cix(any_cix::none); // invalid index
}


void rs_table::get_any_cix (const rs_string&  col_name, any_cix& cix, rs_type& col_type) const {
    col_type = rt_none;
    any_cix cs;
    cix = cs;
    any_cix ca = get_any_cix (col_name);
    if (!ca.is_valid())
        return;
    col_type = rst_table[ca].rsc_type;
    cix = ca;
}